import os
import sys
import re
import commands
import nltk, pprint
from nltk.corpus import stopwords
from nltk.collocations import BigramCollocationFinder
from nltk.metrics import f_measure, BigramAssocMeasures

class Extractor:
	def __init__(self, raw):
		self.raw = raw
		self.tokens = nltk.word_tokenize(self.raw)
		#self.tokens = [w.lower() for w in self.tokens]
		self.text = nltk.Text(self.tokens)
		#A collocation is a sequence of words that occur together unusually often
		self.collocations = self.getCollocations()
	
	def lexical_diversity():
		return len(self.text) / len(set(self.text))

	def percentage(self, word, total):
		return 100 * self.text(word) / len(self.text)

	def getLongWords(self):
		V = set(self.text)
		return [w for w in V if len(w) > 15]
			
	def getCollocations(self):
		window_size = 2
		num=20
		ignored_words = stopwords.words('english')
		finder = BigramCollocationFinder.from_words(self.tokens, window_size)
		finder.apply_freq_filter(2)
		finder.apply_word_filter(lambda w: len(w) < 3 or w.lower() in ignored_words)
		bigram_measures = BigramAssocMeasures()
		collocations = finder.nbest(bigram_measures.likelihood_ratio, num)
		colloc_strings = [w1+' '+w2 for w1, w2 in collocations]
		return colloc_strings

		
		
